import os
NOVA_HOME = "/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA"
NOVA_DATA_HOME = '/home/labs/hornsteinlab/Collaboration/FUNOVA'
LOGS_PATH = os.path.join(NOVA_DATA_HOME, "outputs/preprocessing/logs/")
PLOT_PATH = os.path.join(NOVA_DATA_HOME, "outputs/logs/")
os.chdir(NOVA_HOME)
import pandas as pd
import contextlib
import io
from IPython.display import display, Javascript
import seaborn as sns
from tools.preprocessing_tools.qc_reports.qc_utils import log_files_qc, run_validate_folder_structure, display_diff, sample_and_calc_variance, \
show_site_survival_dapi_brenner, show_site_survival_dapi_cellpose, \
show_site_survival_dapi_tiling, show_site_survival_target_brenner, \
calc_total_sums, plot_filtering_heatmap, show_total_sum_tables, \
plot_cell_count, plot_catplot, plot_hm_combine_batches, plot_hm, \
run_calc_hist_new, plot_marker_data, find_bad_wells, create_marker_info_df, \
show_site_survival_by_brenner_on_dapi_tiles, show_site_survival_target_brenner_tiles
from tools.preprocessing_tools.qc_reports.qc_config import (
funova_markers as markers,
funova_cell_lines as cell_lines,
funova_cell_lines_to_cond as cell_lines_to_cond,
funova_cell_lines_for_disp as cell_lines_for_disp,
funova_reps as reps,
funova_line_colors as line_colors,
funova_lines_order as lines_order,
funova_custom_palette as custom_palette,
funova_expected_dapi_raw as expected_dapi_raw,
funova_panels as panels,
funova_marker_info as marker_info
)
from tools.show_images_utils import *
%load_ext autoreload
%autoreload 2
# choose batches
batches = ['Batch1', 'Batch2']#, 'batch2', 'batch3']
batches
['Batch1', 'Batch2']
validate_antibody = False
df = log_files_qc(LOGS_PATH, batches, only_wt_cond = False)
reading logs of Batch1 Total of 1 files were read. Before dup handeling (171516, 22) After duplication removal #1: (171516, 23) After duplication removal #2: (171516, 23)
df[['cell_line', 'marker', 'cell_line_cond']] = df[['cell_line', 'marker', 'cell_line_cond']].apply(lambda x: x.str.replace('_', '-', regex=False))
df['filename'] = df['filename'].str.split('-').str[0]
df['site_num'] = df['site_num'].str.split('-').str[0]
df_dapi = df[df.marker=='DAPI']
df_target = df[df.marker!='DAPI']
root_directory_raw = os.path.join(NOVA_DATA_HOME, 'input', 'images', 'raw')
df_metadata = extract_image_metadata(root_directory_raw, FILE_EXTENSION = '.tiff', KEY_BATCH = 'Batch')
# root_path = "/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/"
# marker_info_df = create_marker_info_df(root_path)
batches_raw = [batch.replace("_16bit_no_downsample","") for batch in batches]
raws = run_validate_folder_structure(root_directory_raw, False, panels, markers.copy(), PLOT_PATH, marker_info,
cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
batches=batches_raw, fig_width=8,fig_height = 40,
expected_count=100, validate_antibody = validate_antibody)
Batch1 Folder structure is valid. No bad files are found. Total Sites: 118400
======== Batch2 Folder structure is valid. No bad files are found. Total Sites: 118400
======== ====================
root_directory_proc = os.path.join(NOVA_DATA_HOME, 'input', 'images', 'processed')
procs = run_validate_folder_structure(root_directory_proc, True, panels, markers, PLOT_PATH, marker_info,
cell_lines_to_cond, reps, cell_lines_for_disp, expected_dapi_raw,
batches=batches, fig_width=8,fig_height = 40,
expected_count=100, validate_antibody = validate_antibody)
Batch1 Folder structure is valid. No bad files are found. Total Sites: 83439
======== Batch2 Folder structure is valid. No bad files are found. Total Sites: 64207
======== ====================
display_diff(batches, raws, procs, PLOT_PATH, fig_width=8, fig_height = 40)
Batch1
======== Batch2
========
for batch in batches:
with contextlib.redirect_stdout(io.StringIO()):
var = sample_and_calc_variance(root_directory_proc, batch,
sample_size_per_markers=200, cond_count=1, rep_count=len(reps),
num_markers=len(markers))
print(f'{batch} var: ',var)
Batch1 var: 0.021849141482731094 Batch2 var: 0.025001876697681818
By order of filtering
Percentage out of the total sites
dapi_filter_by_brenner = show_site_survival_dapi_brenner(df_dapi,batches, line_colors, panels,
figsize=(6,18), reps=reps, vmax=100)
show_images(df_metadata, marker='DAPI', show_DAPI=False, batch=2, panel='L', cell_line='C9orf72-HRE-1008566', rep=1,
condition='stress', funova=True, samples=5)
1
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f61-ch1t1.tiff 2 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f07-ch1t1.tiff 3 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f81-ch1t1.tiff 4 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f01-ch1t1.tiff 5 Img shape: (1024, 1024)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelL/stress/rep1/DAPI/r12c23f72-ch1t1.tiff
show_images(df_metadata, marker='DAPI', show_DAPI=False, batch=2, panel='D', cell_line='Control-1017118', rep=1,
condition='stress', funova=True, samples=5)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
1 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f66-ch1t1.tiff 2 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f59-ch1t1.tiff 3 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f26-ch1t1.tiff 4 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f67-ch1t1.tiff 5 Img shape: (1024, 1024)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/Control-1017118/panelD/stress/rep1/DAPI/r15c07f46-ch1t1.tiff
# temp1 = get_specific_imgs(dfb, marker='DAPI', batch=2,cell_line='Control_1017118', rep=1, condition='stress')
# temp = temp1.loc[temp1.Panel =='D'].Target_Sharpness_Brenner
# temp1 = temp1.Target_Sharpness_Brenner
# plt.hist(temp1)
# plt.hist(temp)
Percentage out of the sites that passed the previous filter. In parenthesis are absolute values.
A site will be filtered out if Cellpose found 0 cells in it.
dapi_filter_by_cellpose = show_site_survival_dapi_cellpose(df_dapi, batches, dapi_filter_by_brenner,
line_colors, panels, figsize=(6,18), reps=reps)
Percentage out of the sites that passed the previous filter. In parenthesis are absolute values.
A site will be filtered out if after tiling, no tile is containing at least 85% of a cell that Cellpose detected.
dapi_filter_by_tiling=show_site_survival_dapi_tiling(df_dapi, batches, dapi_filter_by_cellpose,
line_colors, panels, figsize=(6,18), reps=reps)
Percentage out of the sites that passed the previous filter. In parenthesis are absolute values (if different than the percentages).
dapi_filter_by_brenner_tiles=show_site_survival_by_brenner_on_dapi_tiles(df_dapi, batches, dapi_filter_by_tiling,
line_colors, panels, figsize=(6,18), reps=reps)
Percentage out of the sites that passed the previous filter. In parenthesis are absolute values (if different than the percentages).
show_site_survival_target_brenner(df_dapi, df_target, dapi_filter_by_brenner_tiles,
figsize=(6,24), markers=markers)
df_dapi.iloc[np.where(df_dapi['n_valid_tiles']-df_dapi['n_valid_tiles_after_tiles_brenner'])][['n_valid_tiles','n_valid_tiles_after_tiles_brenner']]
| n_valid_tiles | n_valid_tiles_after_tiles_brenner | |
|---|---|---|
| 61865 | 2 | 0.0 |
| 61789 | 3 | 2.0 |
| 61609 | 2 | 0.0 |
| 61606 | 1 | 0.0 |
| 61633 | 2 | 0.0 |
| ... | ... | ... |
| 118993 | 1 | 0.0 |
| 118990 | 1 | 0.0 |
| 119014 | 2 | 0.0 |
| 118984 | 1 | 0.0 |
| 119013 | 1 | 0.0 |
5557 rows × 2 columns
## Images that were completely removed and their corresponding DAPI
show_images(df_metadata, marker = 'Senescence-signaling', show_DAPI = True,
batch = 1, cell_line ='Control-1048087',
condition = 'stress', rep=2, funova = True, samples = 5)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
1 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/Senescence-signaling/r09c24f54-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/DAPI/r09c24f54-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 2 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/Senescence-signaling/r09c24f03-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/DAPI/r09c24f03-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 3 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/Senescence-signaling/r09c24f27-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/DAPI/r09c24f27-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 4 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/Senescence-signaling/r09c24f83-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/DAPI/r09c24f83-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 5 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/Senescence-signaling/r09c24f38-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/Control-1048087/panelL/stress/rep2/DAPI/r09c24f38-ch1t1.tiff Img shape: (1024, 1024)
--------------------------------
show_images(df_metadata, marker='Parthanatos-early', show_DAPI=False, batch=2, panel='H', cell_line='TDP--43-G348V-1057052',
rep=2, condition='Untreated', funova=True, samples=5)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
1 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f12-ch2t1.tiff 2 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f18-ch2t1.tiff 3 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f37-ch2t1.tiff 4 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f05-ch2t1.tiff 5 Img shape: (1024, 1024)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/TDP--43-G348V-1057052/panelH/Untreated/rep2/Parthanatos-early/r02c16f23-ch2t1.tiff
show_images(df_metadata, marker='Protein-degradation', show_DAPI=False, batch=2, panel='L', cell_line='C9orf72-HRE-981344', rep=1,
condition='stress', funova=True, samples=5)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
1 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f53-ch2t1.tiff 2 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f78-ch2t1.tiff 3 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f14-ch2t1.tiff 4 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f77-ch2t1.tiff 5 Img shape: (1024, 1024)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-981344/panelL/stress/rep1/Protein-degradation/r16c23f29-ch2t1.tiff
Find non valid wells
dfb = pd.read_csv("/home/labs/hornsteinlab/Collaboration/FUNOVA/outputs/preprocessing/brenner/raw_metrics280125_exp3.csv")
# Extract the panel (e.g., "panelD") from the Path column
dfb['Panel'] = dfb['Path'].str.extract(r'/panel([A-Z])/', expand=True)
dfb['image_id'] = dfb['Path'].apply(lambda path: path.split('/')[-1].split('.')[0])
find_bad_wells(dfb, threshold=50000, percentage_filter=70)
| Panel | Condition | CellLine | Rep | Batch | Count_table1 | Count_table2 | Percentage | Combination | Marker | |
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | J | stress | C9orf72-HRE_981344 | rep1 | Batch1 | 221 | 300 | 73.666667 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | NaN |
| 1 | L | stress | C9orf72-HRE_1008566 | rep1 | Batch2 | 285 | 300 | 95.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | NaN |
| 2 | L | stress | C9orf72-HRE_981344 | rep1 | Batch2 | 268 | 300 | 89.333333 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | NaN |
| 3 | NaN | Untreated | TDP--43-G348V_1057052 | rep2 | Batch2 | 84 | 100 | 84.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | Parthanatos-early |
| 4 | NaN | stress | C9orf72-HRE_1008566 | rep1 | Batch2 | 96 | 100 | 96.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | Protein-degradation |
| 5 | NaN | stress | C9orf72-HRE_1008566 | rep1 | Batch2 | 94 | 100 | 94.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | Senescence-signaling |
| 6 | NaN | stress | C9orf72-HRE_981344 | rep1 | Batch1 | 91 | 100 | 91.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | DNA-damage-P53BP1 |
| 7 | NaN | stress | C9orf72-HRE_981344 | rep1 | Batch1 | 77 | 100 | 77.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | Neuronal-activity |
| 8 | NaN | stress | C9orf72-HRE_981344 | rep1 | Batch2 | 90 | 100 | 90.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | Protein-degradation |
| 9 | NaN | stress | C9orf72-HRE_981344 | rep1 | Batch2 | 91 | 100 | 91.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | Senescence-signaling |
| 10 | NaN | stress | C9orf72-HRE_981344 | rep2 | Batch1 | 78 | 100 | 78.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | Nuclear-speckles-SC35 |
| 11 | NaN | stress | C9orf72-HRE_981344 | rep2 | Batch2 | 79 | 100 | 79.000000 | ['Condition', 'CellLine', 'Rep', 'Batch', 'Mar... | DNA-damage-P53BP1 |
| 12 | D | stress | Control_1017118 | rep1 | Batch2 | 92 | 100 | 92.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | DAPI |
| 13 | F | stress | C9orf72-HRE_981344 | rep2 | Batch1 | 78 | 100 | 78.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | Nuclear-speckles-SC35 |
| 14 | H | Untreated | TDP--43-G348V_1057052 | rep2 | Batch2 | 84 | 100 | 84.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | Parthanatos-early |
| 15 | J | stress | C9orf72-HRE_981344 | rep1 | Batch1 | 91 | 100 | 91.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | DNA-damage-P53BP1 |
| 16 | J | stress | C9orf72-HRE_981344 | rep1 | Batch1 | 77 | 100 | 77.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | Neuronal-activity |
| 17 | J | stress | C9orf72-HRE_981344 | rep2 | Batch2 | 79 | 100 | 79.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | DNA-damage-P53BP1 |
| 18 | L | stress | C9orf72-HRE_1008566 | rep1 | Batch2 | 95 | 100 | 95.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | DAPI |
| 19 | L | stress | C9orf72-HRE_1008566 | rep1 | Batch2 | 96 | 100 | 96.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | Protein-degradation |
| 20 | L | stress | C9orf72-HRE_1008566 | rep1 | Batch2 | 94 | 100 | 94.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | Senescence-signaling |
| 21 | L | stress | C9orf72-HRE_981344 | rep1 | Batch2 | 87 | 100 | 87.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | DAPI |
| 22 | L | stress | C9orf72-HRE_981344 | rep1 | Batch2 | 90 | 100 | 90.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | Protein-degradation |
| 23 | L | stress | C9orf72-HRE_981344 | rep1 | Batch2 | 91 | 100 | 91.000000 | ['Panel', 'Condition', 'CellLine', 'Rep', 'Bat... | Senescence-signaling |
# thresholds = {
# "Stress_initiation": (0, 99),
# "Aberrant_splicing": (1, 97),
# "Autophagy": (5, 97),
# "DAPI": (0.2, 95),
# "Apoptosis": (0.27, 95),
# "impaired_Autophagosome": (5, 99),
# "Cytoskeleton": (10, 99.8),
# "DNA_damage_P53BP1": (0.3, 90),
# "DNA_damage_pH2Ax": (7, 85),
# "mature_Autophagosome": (2, 98),
# "Necrosis": (4, 95),
# "Neuronal_activity": (5, 90),
# "Nuclear_speckles_SC35": (0.2, 85),
# "Nuclear_speckles_SON": (0.2, 85),
# "Parthanatos_early": (0, 85),
# "Parthanatos_late": (5, 95),
# "Protein_degradation": (5, 92),
# "Senescence_signaling": (10, 90),
# "Splicing_factories": (0, 95),
# "TDP-43": (5, 99.2),
# "Ubiquitin_levels": (2, 90),
# "UPR_ATF4": (2, 99.7),
# "UPR_ATF6": (3, 100),
# "UPR_IRE1a": (2, 98),
# "Necroptosis_pMLKL": (5, 90),
# "Necroptosis_HMGB1": (0.01, 90),
# }
# # Extended batch data with new entries
# data = [
# (2, 'DNA_damage_pH2Ax', 1, 1017118, 'Untreated'),
# (2, 'Parthanatos_early', 1, 1048087, 'stress'),
# (2, 'DNA_damage_pH2Ax', 2, 1008566, 'stress'),
# (2, 'Parthanatos_early', 1, 981344, 'stress'),
# (2, 'Nuclear_speckles_SON', 1, 981344, 'stress'),
# (1, 'Autophagy', 1, 1057052, 'stress'),
# (1, 'impaired_Autophagosome', 1, 1057052, 'stress'),
# (1, 'UPR_IRE1a', 1, 1057052, 'stress'),
# (1, 'Ubiquitin_levels', 1, 1057052, 'stress'),
# (1, 'UPR_IRE1a', 1, 981344, 'stress'),
# (1, 'Ubiquitin_levels', 1, 981344, 'stress'),
# (1, 'DNA_damage_P53BP1', 1, 981344, 'stress'),
# (1, 'Neuronal_activity', 1, 981344, 'stress'),
# (1, 'Necroptosis_HMGB1', 1, 981344, 'stress'),
# (1, 'Necrosis', 1, 981344, 'stress'),
# (1, 'Nuclear_speckles_SC35', 2, 981344, 'stress'),
# (1, 'Splicing_factories', 2, 981344, 'stress'),
# (1, 'Aberrant_splicing', 2, 981344, 'stress'),
# (1, 'Parthanatos_late', 2, 981344, 'stress'),
# (1, 'Senescence_signaling', 2, 1048087, 'stress'),
# (1, 'Apoptosis', 2, 981344, 'stress'),
# (1, 'Necroptosis_pMLKL', 2, 981344, 'stress'),
# (2, 'Autophagy', 1, 1017118, 'stress'),
# (2, 'impaired_Autophagosome', 1, 1017118, 'stress'),
# (2, 'UPR_IRE1a', 1, 1048087, 'Untreated'),
# (2, 'UPR_IRE1a', 1, 1008566, 'stress'),
# (2, 'Ubiquitin_levels', 1, 1048087, 'Untreated'),
# (2, 'Ubiquitin_levels', 1, 1008566, 'stress'),
# (2, 'DNA_damage_P53BP1', 1, 1017118, 'stress'),
# (2, 'Neuronal_activity', 1, 1017118, 'stress'),
# (2, 'DNA_damage_P53BP1', 2, 981344, 'stress'),
# (2, 'Neuronal_activity', 2, 981344, 'stress'),
# (2, 'DNA_damage_P53BP1', 1, 981344, 'stress'),
# (2, 'Neuronal_activity', 1, 981344, 'stress'),
# (2, 'DNA_damage_P53BP1', 1, 1057052, 'stress'),
# (2, 'Neuronal_activity', 1, 1057052, 'stress'),
# (2, 'DNA_damage_P53BP1', 1, 1005373, 'stress'),
# (2, 'Neuronal_activity', 1, 1005373, 'stress'),
# (2, 'DNA_damage_P53BP1', 2, 1057052, 'stress'),
# (2, 'Neuronal_activity', 2, 1057052, 'stress'),
# (2, 'Necroptosis_HMGB1', 1, 981344, 'Untreated'),
# (2, 'Necrosis', 1, 981344, 'Untreated'),
# (2, 'Necroptosis_HMGB1', 2, 981344, 'stress'),
# (2, 'Necrosis', 2, 981344, 'stress'),
# (2, 'Necroptosis_HMGB1', 2, 1001733, 'stress'),
# (2, 'Necrosis', 2, 1001733, 'stress'),
# (2, 'Necroptosis_HMGB1', 2, 1005373, 'stress'),
# (2, 'Necrosis', 2, 1005373, 'stress')
# ]
# # Cell line ID mapping
# cell_line_mapping = {
# 1008566: 'C9orf72-HRE_1008566',
# 981344: 'C9orf72-HRE_981344',
# 1001733: 'Control_1001733',
# 1017118: 'Control_1017118',
# 1025045: 'Control_1025045',
# 1048087: 'Control_1048087',
# 1057052: 'TDP--43-G348V_1057052',
# 1005373: 'TDP--43-N390D_1005373'
# }
# # Convert to DataFrame (renamed to `marker_df`)
# marker_df = pd.DataFrame(data, columns=['Batch', 'Name', 'Rep', 'CellLineID', 'Condition'])
# lower_thresh_DAPI, upper_thresh_DAPI = thresholds['DAPI']
# cb = get_specific_imgs(dfb, marker='DAPI')
# cb = cb.loc[(cb.Batch == 'Batch1') | (cb.Batch == 'Batch2')].Target_Sharpness_Brenner.values
# lower_p_DAPI = np.percentile(cb, lower_thresh_DAPI)
# upper_p_DAPI = np.percentile(cb, upper_thresh_DAPI)
# # Iterate over each row
# for _, row in marker_df.iterrows():
# batch, marker, rep, cell_line_id, condition = row
# # Match Cell Line Name
# cell_line = cell_line_mapping.get(cell_line_id, f'Unknown_{cell_line_id}')
# # Fetch images based on specific conditions
# cb1 = get_specific_imgs(dfb, marker=marker, batch=batch, rep=rep,
# cell_line=cell_line, condition=condition)
# dapi_paths = cb1.Path.apply(lambda p: get_dapi_path(p, marker1=marker, marker2='DAPI', funova=True))
# dfDAPI = dfb.loc[dfb['Path'].isin(dapi_paths)].Target_Sharpness_Brenner.values
# cb1 = cb1.Target_Sharpness_Brenner.values
# # Print statistics dynamically for the specific selection
# print(f"Stats for marker: {marker}, batch: {batch}, rep: {rep}, cell_line: {cell_line}, condition: {condition}")
# print(f"Mean: {np.mean(cb1):.2f}, Median: {np.median(cb1):.2f}, "
# f"10th Percentile: {np.percentile(cb1, 10):.2f}, "
# f"50th Percentile (Median): {np.percentile(cb1, 50):.2f}, "
# f"90th Percentile: {np.percentile(cb1, 90):.2f}")
# # Fetch images for the entire dataset with the given marker
# cb = get_specific_imgs(dfb, marker=marker)
# cb = cb.loc[(cb.Batch == 'Batch1') | (cb.Batch == 'Batch2')].Target_Sharpness_Brenner.values
# lower_thresh, upper_thresh = thresholds[marker]
# lower_p = np.percentile(cb, lower_thresh)
# upper_p = np.percentile(cb, upper_thresh)
# filtered_out = np.sum((cb1 < lower_p) | (cb1 > upper_p))
# total_count = len(cb1)
# filtered_percentage = (filtered_out / total_count) * 100 if total_count > 0 else 0
# filtered_out_DAPI = np.sum((dfDAPI < lower_p_DAPI) | (dfDAPI > upper_p_DAPI))
# total_count_DAPI = len(dfDAPI)
# filtered_percentage_DAPI = (filtered_out_DAPI / total_count_DAPI) * 100 if total_count_DAPI > 0 else 0
# # Print statistics dynamically for the full dataset
# print(f"Stats for marker: {marker} (All batches, reps, and cell lines)")
# print(f"Mean: {np.mean(cb):.2f}, Median: {np.median(cb):.2f}, "
# f"10th Percentile: {np.percentile(cb, 10):.2f}, "
# f"50th Percentile (Median): {np.percentile(cb, 50):.2f}, "
# f"90th Percentile: {np.percentile(cb, 90):.2f}")
# print(f"Threshold Percentiles ({lower_thresh}% - {upper_thresh}%): {lower_p:.2f} - {upper_p:.2f}")
# print(f"Filtered Out: {filtered_out}/{total_count} ({filtered_percentage:.2f}%)")
# print(f"Filtered Out_DAPI: {filtered_out_DAPI}/{total_count_DAPI} ({filtered_percentage_DAPI:.2f}%)")
# print("-" * 80) # Separator for readability
show_images(df_metadata, marker='DNA-damage-pH2Ax', show_DAPI=True, batch=2, cell_line='C9orf72-HRE-1008566', rep=2,
condition='stress', funova=True, samples=5)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
1 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelH/stress/rep2/DNA-damage-pH2Ax/r12c16f42-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/C9orf72-HRE-1008566/panelH/stress/rep2/DAPI/r12c16f42-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 2 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelH/stress/rep2/DNA-damage-pH2Ax/r12c16f20-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/C9orf72-HRE-1008566/panelH/stress/rep2/DAPI/r12c16f20-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 3 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelH/stress/rep2/DNA-damage-pH2Ax/r12c16f52-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/C9orf72-HRE-1008566/panelH/stress/rep2/DAPI/r12c16f52-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 4 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelH/stress/rep2/DNA-damage-pH2Ax/r12c16f89-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/C9orf72-HRE-1008566/panelH/stress/rep2/DAPI/r12c16f89-ch1t1.tiff Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
-------------------------------- 5 Img shape: (1024, 1024)
WARNING:root:The image shape (1080, 1080) isn't as expected (1024, 1024). Cropping image to fit WARNING:root:Removing pixels from the frame (28, 28)
/home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch2/C9orf72-HRE-1008566/panelH/stress/rep2/DNA-damage-pH2Ax/r12c16f98-ch3t1.tiff /home/labs/hornsteinlab/Collaboration/FUNOVA/input/images/raw/Batch1/C9orf72-HRE-1008566/panelH/stress/rep2/DAPI/r12c16f98-ch1t1.tiff Img shape: (1024, 1024)
--------------------------------
Percentage out of the tiles that passed the previous filter. In parenthesis are absolute values (if different than the percentages).
show_site_survival_target_brenner_tiles(df_dapi, df_target, dapi_filter_by_brenner_tiles,
figsize=(6,24), markers=markers)
names = ['Total number of tiles', 'Total number of whole cells']
stats = ['n_valid_tiles_after_tiles_brenner','site_whole_cells_counts_sum','site_cell_count','site_cell_count_sum']
total_sum = calc_total_sums(df_target, df_dapi, stats, markers)
total_sum.rename(columns={'n_valid_tiles_after_tiles_brenner': 'n_valid_tiles'}, inplace=True)
plot_marker_data(total_sum, split_by_cell_line=True)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
plot_marker_data(total_sum, split_by_cell_line=False)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1351: FutureWarning: The `ci` parameter is deprecated. Use `errorbar=None` for the same effect. sns.barplot(
total_sum.n_valid_tiles.sum()
759888.0
total_sum[total_sum.marker =='DAPI'].site_whole_cells_counts_sum.sum()
316044.0
total_sum[total_sum.marker =='DAPI'].site_cell_count.sum()
930452.0
show_total_sum_tables(total_sum)
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| Batch1 | ||||
| count | 1179.000000 | 1179.000000 | 1179.000000 | 1.179000e+03 |
| mean | 381.112807 | 3.811128 | 459.890585 | 1.349077e+03 |
| std | 222.798919 | 2.227989 | 292.262643 | 9.472279e+02 |
| min | 0.000000 | 0.000000 | 2.000000 | 5.000000e+00 |
| 25% | 217.000000 | 2.170000 | 249.000000 | 6.580000e+02 |
| 50% | 355.000000 | 3.550000 | 413.000000 | 1.149000e+03 |
| 75% | 520.500000 | 5.205000 | 598.500000 | 1.777000e+03 |
| max | 1152.000000 | 11.520000 | 1527.000000 | 4.865000e+03 |
| sum | 449332.000000 | NaN | 542211.000000 | 1.590562e+06 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 4.500000e+02 |
| n_valid_tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| Batch2 | ||||
| count | 1171.000000 | 1171.000000 | 1171.000000 | 1.171000e+03 |
| mean | 265.205807 | 2.652058 | 327.519214 | 9.512963e+02 |
| std | 217.176240 | 2.171762 | 281.041345 | 8.970028e+02 |
| min | 0.000000 | 0.000000 | 0.000000 | 0.000000e+00 |
| 25% | 101.500000 | 1.015000 | 120.000000 | 3.030000e+02 |
| 50% | 207.000000 | 2.070000 | 247.000000 | 6.720000e+02 |
| 75% | 395.000000 | 3.950000 | 464.000000 | 1.368000e+03 |
| max | 1177.000000 | 11.770000 | 1576.000000 | 4.868000e+03 |
| sum | 310556.000000 | NaN | 383525.000000 | 1.113968e+06 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 4.500000e+02 |
| n valid tiles | % valid tiles | site_whole_cells_counts_sum | site_cell_count | |
|---|---|---|---|---|
| All batches | ||||
| count | 2350.000000 | 2350.000000 | 2350.000000 | 2.350000e+03 |
| mean | 323.356596 | 3.233566 | 393.930213 | 1.150864e+03 |
| std | 227.477570 | 2.274776 | 294.209390 | 9.435553e+02 |
| min | 0.000000 | 0.000000 | 0.000000 | 0.000000e+00 |
| 25% | 146.000000 | 1.460000 | 170.250000 | 4.430000e+02 |
| 50% | 282.000000 | 2.820000 | 329.000000 | 8.950000e+02 |
| 75% | 452.000000 | 4.520000 | 548.000000 | 1.593750e+03 |
| max | 1177.000000 | 11.770000 | 1576.000000 | 4.868000e+03 |
| sum | 759888.000000 | NaN | 925736.000000 | 2.704530e+06 |
| expected_count | 450.000000 | 450.000000 | 450.000000 | 4.500000e+02 |
For each batch, cell line, replicate and markerTotal number of tiles
to_heatmap = total_sum.rename(columns={'n_valid_tiles':'index'})
plot_filtering_heatmap(to_heatmap, extra_index='marker', vmin=None, vmax=None,
xlabel = 'Total number of tiles', show_sum=True, figsize=(6,24))
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10) /home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10) /home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
For each batch, cell line, replicate and markerTotal number of tiles
to_heatmap = total_sum.rename(columns={'site_whole_cells_counts_sum':'index'})
plot_filtering_heatmap(to_heatmap, extra_index='marker', vmin=None, vmax=None,
xlabel = 'Total number of whole cells', show_sum=True, figsize=(6,24))
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10) /home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10) /home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:388: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator. ax.set_yticklabels(ax.get_yticklabels(), fontsize=10)
df_no_empty_sites = df_dapi[df_dapi.n_valid_tiles_after_tiles_brenner !=0]
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_cell_count_sum',
title='Cell Count Average per Site (from tiles)', figsize=(16,6))
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_whole_cells_counts_sum',
title='Whole Cell Count Average per Site',figsize=(16,6))
plot_cell_count(df_no_empty_sites, lines_order, custom_palette, y='site_cell_count',
title='Cellpose Cell Count Average per Site',figsize=(16,6))
df_dapi.groupby(['cell_line_cond']).n_valid_tiles_after_tiles_brenner.mean()
cell_line_cond C9orf72-HRE-1008566 Untreated 2.332305 C9orf72-HRE-1008566 stress 2.387565 C9orf72-HRE-981344 Untreated 2.127250 C9orf72-HRE-981344 stress 1.184750 Control-1001733 Untreated 4.174963 Control-1001733 stress 3.239484 Control-1017118 Untreated 8.559741 Control-1017118 stress 5.880845 Control-1025045 Untreated 5.446470 Control-1025045 stress 3.983206 Control-1048087 Untreated 4.160327 Control-1048087 stress 3.700133 TDP--43-G348V-1057052 Untreated 1.543040 TDP--43-G348V-1057052 stress 1.027178 TDP--43-N390D-1005373 Untreated 3.548238 TDP--43-N390D-1005373 stress 2.486610 Name: n_valid_tiles_after_tiles_brenner, dtype: float64
plot_catplot(df_dapi, sns.color_palette('colorblind'), reps=reps,x='cell_line', y_title='Valid Tiles Count', x_title='Cell Line', y='n_valid_tiles_after_tiles_brenner', hue='rep',
height=4, aspect=2)
/home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1028: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy df.loc[:, 'batch_rep'] = df['batch'] + " " + df['rep'] /home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/tools/preprocessing_tools/qc_reports/qc_utils.py:1039: UserWarning: The palette list has more values (10) than needed (2), which may not be intended. g = sns.catplot(kind='box', data=df, y=y, x=x,height=height, aspect=aspect, hue=hue, palette=palette,
# plot_hm(df_dapi, split_by='rep', rows='cell_line', columns='panel', vmax=3)
# for batch in batches:
# print(batch)
# run_calc_hist_new(batch,cell_lines_for_disp, markers, root_directory_raw, root_directory_proc,
# hist_sample=10,sample_size_per_markers=10, ncols=4, nrows=1, figsize=(6,2))
# print("="*30)
# save notebook as HTML ( the HTML will be saved in the same folder the original script is)
display(Javascript('IPython.notebook.save_checkpoint();'))
os.system(f'jupyter nbconvert --to html tools/preprocessing_tools/qc_reports/qc_report_funova_exp3.ipynb --output {NOVA_HOME}/manuscript/preprocessing_qc_reports/qc_report_funova_exp3_11.03.25.html')
[NbConvertApp] Converting notebook tools/preprocessing_tools/qc_reports/qc_report_funova_exp3.ipynb to html [NbConvertApp] Writing 43272366 bytes to /home/labs/hornsteinlab/Collaboration/NOVA_GAL/NOVA/manuscript/preprocessing_qc_reports/qc_report_funova_exp3_11.03.25.html
0